import matplotlib.pyplot as plt
keywords = [
    "Traffic Management",
    "Safety",
    "Resource Allocation",
    "Data Decisions",
    "Economic Impact",
    "Environmental Benefits",
    "Public Satisfaction",
    "Smart City"
]
fig, ax = plt.subplots(figsize=(8, 4))
labels = keywords
sizes = [1] * len(keywords)
colors = ['skyblue', 'lightcoral', 'lightgreen', 'lightyellow', 'lightsalmon', 'lightpink', 'lightblue', 'lightgrey']
explode = [0.1] * len(keywords)

ax.pie(sizes, explode=explode, labels=labels, colors=colors, shadow=True, startangle=140)
ax.axis('equal')
plt.title('Why is it useful to do this analysis?', fontsize=14, fontweight='bold')
plt.show()

import pandas as pd
from matplotlib import pyplot as plt
import seaborn as sns
import joblib

file_path = '~/Downloads/RTA Dataset.csv'
data = pd.read_csv(file_path)
data.head()

data_summary = pd.DataFrame({
    'Data Type': data.dtypes,
    'Missing Values': data.isnull().sum(),
    'Unique Values': data.nunique()
})
data_summary = data_summary.sort_values(by='Missing Values', ascending=False)
data_summary

data_summary.groupby('Data Type').size().plot(kind='barh', color=sns.palettes.mpl_palette('Dark2'))
plt.gca().spines[['top', 'right',]].set_visible(False)

figsize = (12, 1.2 * len(data_summary['Data Type'].unique()))
plt.figure(figsize=figsize)
sns.violinplot(data_summary, x='Missing Values', hue='Data Type', inner='stick', palette='Dark2')
sns.despine(top=True, right=True, bottom=True, left=True)

plt.figure(figsize=(8,4))
colours = ['#34495E', 'seagreen']
sns.heatmap(data.isnull(), cmap=sns.color_palette(colours))

<Axes: >

missing_values = data.isnull().sum()
print(missing_values[missing_values > 0])

Educational_level           741
Vehicle_driver_relation     579
Driving_experience          829
Type_of_vehicle             950
Owner_of_vehicle            482
Service_year_of_vehicle    3928
Defect_of_vehicle          4427
Area_accident_occured       239
Lanes_or_Medians            385
Road_allignment             142
Types_of_Junction           887
Road_surface_type           172
Type_of_collision           155
Vehicle_movement            308
Work_of_casuality          3198
Fitness_of_casuality       2635
dtype: int64

from scipy.stats import chi2_contingency
import warnings

data['Accident_severity'] = pd.Categorical(data['Accident_severity'], categories=['Slight Injury', 'Serious Injury', 'Fatal injury'])

columns_to_analyze = ['Service_year_of_vehicle', 'Defect_of_vehicle', 'Work_of_casuality', 'Fitness_of_casuality']

plt.figure(figsize=(5, 5))
unique_severities = data['Accident_severity'].unique()
palette = sns.color_palette("Set2", len(unique_severities))

# Suppress FutureWarnings from seaborn
warnings.filterwarnings('ignore', category=FutureWarning)

# Plot the distribution of 'Accident_severity' with and without missing values
fig, axs = plt.subplots(len(columns_to_analyze), 2, figsize=(10, 5 * len(columns_to_analyze)))

for i, column in enumerate(columns_to_analyze):
    sns.countplot(x='Accident_severity', data=data[data[column].isnull()], ax=axs[i, 0], palette=palette)
    axs[i, 0].set_title(f'Accident Severity with Missing {column}')
    axs[i, 0].legend(labels=unique_severities, loc='upper right')

    sns.countplot(x='Accident_severity', data=data[data[column].notnull()], ax=axs[i, 1], palette=palette)
    axs[i, 1].set_title(f'Accident Severity without Missing {column}')
    axs[i, 1].legend(labels=unique_severities, loc='upper right')

plt.tight_layout()
plt.show()

# Perform chi-squared tests
chi2_results = {}
for column in columns_to_analyze:
    contingency_table = pd.crosstab(data['Accident_severity'], data[column].isnull())
    chi2, p, dof, expected = chi2_contingency(contingency_table)
    chi2_results[column] = {'chi2': chi2, 'p': p}

chi2_results

<Figure size 500x500 with 0 Axes>

{'Service_year_of_vehicle': {'chi2': 1.722305458919597,
  'p': 0.422674571956952},
 'Defect_of_vehicle': {'chi2': 2.8583048690672266, 'p': 0.2395118382021787},
 'Work_of_casuality': {'chi2': 0.07546190767365601, 'p': 0.9629719899606989},
 'Fitness_of_casuality': {'chi2': 0.37391140712037346, 'p': 0.82948047860372}}

# Impute with the most frequent value for 'Service_year_of_vehicle' and 'Work_of_casuality'
most_frequent_service_year = data['Service_year_of_vehicle'].mode()[0]
data['Service_year_of_vehicle'].fillna(most_frequent_service_year, inplace=True)

most_frequent_work_of_casuality = data['Work_of_casuality'].mode()[0]
data['Work_of_casuality'].fillna(most_frequent_work_of_casuality, inplace=True)

# Impute with a placeholder for 'Defect_of_vehicle' and 'Fitness_of_casuality'
data['Defect_of_vehicle'].fillna('Unknown', inplace=True)
data['Fitness_of_casuality'].fillna('Unknown', inplace=True)

# Handle missing values
# Drop columns with too many missing values
#columns_to_drop = ['Service_year_of_vehicle', 'Defect_of_vehicle', 'Work_of_casuality', 'Fitness_of_casuality']
#data.drop(columns=columns_to_drop, inplace=True)

data.shape

(12316, 32)

# Impute missing values in other columns with the mode
columns_to_impute = data.columns[data.isnull().sum() > 0]
for column in columns_to_impute:
    mode_value = data[column].mode()[0]
    data[column].fillna(mode_value, inplace=True)

# Verify that there are no missing values left
missing_values_cleaned = data.isnull().sum()
print(missing_values_cleaned[missing_values_cleaned > 0])

Series([], dtype: int64)

fig, axes = plt.subplots(1, 2, figsize=(10, 4))

sns.histplot(data=data, x='Number_of_vehicles_involved', hue='Accident_severity', multiple="stack", ax=axes[0])
axes[0].set_title('Distribution of Number of vehicles involved')
axes[0].set_xlabel('Number of vehicles involved')
axes[0].set_ylabel('Count')

sns.histplot(data=data, x='Number_of_casualties', hue='Accident_severity', multiple="stack", ax=axes[1])
axes[1].set_title('Distribution of Number of casualties')
axes[1].set_xlabel('Number of casualties')
axes[1].set_ylabel('Count')
plt.tight_layout()
plt.show()

import numpy as np
# Apply log transformation in place
data['Number_of_vehicles_involved'] = np.log(data['Number_of_vehicles_involved'] + 1)
data['Number_of_casualties'] = np.log(data['Number_of_casualties'] + 1)


fig, axes = plt.subplots(1, 2, figsize=(10, 4))

sns.histplot(data=data, x='Number_of_vehicles_involved', hue='Accident_severity', multiple="stack", ax=axes[0])
axes[0].set_title('Distribution of Number of Vehicles Involved (Log Transformed)')
axes[0].set_xlabel('Log(Number of Vehicles Involved)')
axes[0].set_ylabel('Count')

sns.histplot(data=data, x='Number_of_casualties', hue='Accident_severity', multiple="stack", ax=axes[1])
axes[1].set_title('Distribution of Number of Casualties (Log Transformed)')
axes[1].set_xlabel('Log(Number of Casualties)')
axes[1].set_ylabel('Count')

plt.tight_layout()
plt.show()

hue_variable = 'Accident_severity'
categorical_columns = data.select_dtypes(include=['object', 'category']).columns
categorical_columns = [col for col in categorical_columns if col != 'Time']
for col in categorical_columns:
    if col != hue_variable:  # Avoid using the hue variable as a category itself
        plt.figure(figsize=(6, 4))
        sns.countplot(y=data[col], hue=data[hue_variable], palette="viridis")
        plt.title(f'Distribution of {col} by {hue_variable}')
        plt.xlabel('Count')
        plt.ylabel(col)
        plt.legend(title=hue_variable)
        plt.show()

# Check for Outliers
numerical_columns = data.select_dtypes(include=['float64', 'int64']).columns
categorical_columns = data.select_dtypes(include=['object', 'category']).columns
fig, axes = plt.subplots(len(numerical_columns)//2, 2, figsize=(12, len(numerical_columns)*2))
axes = axes.flatten()
for i, col in enumerate(numerical_columns):
    sns.boxplot(data=data[col], ax=axes[i], color='skyblue')
    axes[i].set_title(f'Boxplot of {col}')
    axes[i].set_xlabel(col)
plt.tight_layout()
plt.show()

# Analyze Correlations
corr = data[numerical_columns].corr()
plt.figure(figsize=(5, 5))
sns.heatmap(corr, annot=True, cmap='coolwarm', fmt=".2f")
plt.title('Correlation Matrix')
plt.show()

!pip install ydata-profiling > /dev/null 2>&1

from pandas_profiling import ProfileReport
profile = ProfileReport(data, title="Traffic Incident Data Profile Report", explorative=True)
profile.to_notebook_iframe()

/var/folders/2w/vykj3cf54pv9n2rkprsn2ts00000gn/T/ipykernel_43427/841217043.py:1: DeprecationWarning: `import pandas_profiling` is going to be deprecated by April 1st. Please use `import ydata_profiling` instead.
  from pandas_profiling import ProfileReport

# Drop highly correlated features based on the analysis
features_to_drop = ['Casualty_class', 'Road_surface_conditions', 'Age_band_of_casualty','Sex_of_casualty']
data.drop(columns=features_to_drop, inplace=True)

profile = ProfileReport(data, title="Traffic Incident Data Profile Report", explorative=True)
profile.to_notebook_iframe()

# Parse 'Time' column and extract the hour
data['Time'] = pd.to_datetime(data['Time'], format='%H:%M:%S', errors='coerce')
data['hour'] = data['Time'].dt.hour
data['hour'] = data['hour'].fillna(-1)

data.head()

print(data[['Time', 'hour']].head())

                 Time  hour
0 1900-01-01 17:02:00    17
1 1900-01-01 17:02:00    17
2 1900-01-01 17:02:00    17
3 1900-01-01 01:06:00     1
4 1900-01-01 01:06:00     1

data.columns

Index(['Time', 'Day_of_week', 'Age_band_of_driver', 'Sex_of_driver',
       'Educational_level', 'Vehicle_driver_relation', 'Driving_experience',
       'Type_of_vehicle', 'Owner_of_vehicle', 'Service_year_of_vehicle',
       'Defect_of_vehicle', 'Area_accident_occured', 'Lanes_or_Medians',
       'Road_allignment', 'Types_of_Junction', 'Road_surface_type',
       'Light_conditions', 'Weather_conditions', 'Type_of_collision',
       'Number_of_vehicles_involved', 'Number_of_casualties',
       'Vehicle_movement', 'Casualty_severity', 'Work_of_casuality',
       'Fitness_of_casuality', 'Pedestrian_movement', 'Cause_of_accident',
       'Accident_severity', 'hour'],
      dtype='object')

# One-hot encode categorical variables
categorical_columns = data.select_dtypes(include=['object', 'category']).columns
data_encoded = pd.get_dummies(data, columns=categorical_columns, drop_first=True)

# Define target and features
severity_columns = [col for col in data_encoded.columns if 'Accident_severity_' in col]
X = data_encoded.drop(severity_columns + ['Time'], axis=1)  # Ensure 'Time' is not in the features
y = data_encoded[severity_columns]

# Convert target variable to single categorical column
y = y.idxmax(axis=1).apply(lambda x: x.replace('Accident_severity_', ''))

data_encoded.shape

(12316, 171)

data_encoded.columns

Index(['Time', 'Number_of_vehicles_involved', 'Number_of_casualties', 'hour',
       'Day_of_week_Monday', 'Day_of_week_Saturday', 'Day_of_week_Sunday',
       'Day_of_week_Thursday', 'Day_of_week_Tuesday', 'Day_of_week_Wednesday',
       ...
       'Cause_of_accident_No priority to vehicle', 'Cause_of_accident_Other',
       'Cause_of_accident_Overloading', 'Cause_of_accident_Overspeed',
       'Cause_of_accident_Overtaking', 'Cause_of_accident_Overturning',
       'Cause_of_accident_Turnover', 'Cause_of_accident_Unknown',
       'Accident_severity_Serious Injury', 'Accident_severity_Fatal injury'],
      dtype='object', length=171)

from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectFromModel
from imblearn.over_sampling import SMOTE
# Handle imbalanced data using SMOTE
smote = SMOTE(random_state=42)
X_resampled, y_resampled = smote.fit_resample(X, y)

# Convert target labels to numerical format using LabelEncoder
label_encoder = LabelEncoder()
y_resampled_encoded = label_encoder.fit_transform(y_resampled)

# Feature selection using Random Forest
rf = RandomForestClassifier(random_state=42)
rf.fit(X_resampled, y_resampled_encoded)

selector = SelectFromModel(rf, threshold="mean", prefit=True)
support_mask = selector.get_support()

# Select features from the original DataFrame using the support mask
X_selected = X_resampled.loc[:, support_mask]

# Check the shape of the selected features and their names
print("Original shape:", X_resampled.shape)
print("Selected shape:", X_selected.shape)

Original shape: (24316, 168)
Selected shape: (24316, 49)

X_selected.head()

from sklearn.preprocessing import StandardScaler
# Standardize the features
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X_selected)

y_resampled_encoded

array([1, 1, 1, ..., 0, 0, 0])

# Split data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y_resampled_encoded, test_size=0.2, random_state=42)
X_scaled_df=pd.DataFrame(X_scaled)
X_test_df = pd.DataFrame(X_test, columns=X_scaled_df.columns)
y_test_df = pd.Series(y_test, name='target')
X_test_df.to_csv('X_test.csv', index=False)
y_test_df.to_csv('y_test.csv', index=False)

from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import classification_report, accuracy_score

# Initialize the model
nb_model = GaussianNB()

# Fit the model on the training data
nb_model.fit(X_train, y_train)

# Predict and evaluate
nb_pred = nb_model.predict(X_test)
print("Naive Bayes Classifier")
print(classification_report(y_test, nb_pred))
print(f'Accuracy: {accuracy_score(y_test, nb_pred)}')

Naive Bayes Classifier
              precision    recall  f1-score   support

           0       0.88      0.93      0.91      2431
           1       0.92      0.88      0.90      2433

    accuracy                           0.90      4864
   macro avg       0.90      0.90      0.90      4864
weighted avg       0.90      0.90      0.90      4864

Accuracy: 0.9031661184210527

from sklearn.linear_model import LogisticRegression

# Train Logistic Regression
lr_model = LogisticRegression(max_iter=1000, random_state=42)
lr_model.fit(X_train, y_train)

# Predict and evaluate
lr_pred = lr_model.predict(X_test)
print("Logistic Regression")
print(classification_report(y_test, lr_pred))
print(f'Accuracy: {accuracy_score(y_test, lr_pred)}')

Logistic Regression
              precision    recall  f1-score   support

           0       0.99      0.98      0.98      2431
           1       0.98      0.99      0.98      2433

    accuracy                           0.98      4864
   macro avg       0.98      0.98      0.98      4864
weighted avg       0.98      0.98      0.98      4864

Accuracy: 0.9845805921052632

from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import classification_report, accuracy_score

# Train Decision Tree
dt_model = DecisionTreeClassifier(random_state=42)
dt_model.fit(X_train, y_train)

# Predict and evaluate
dt_pred = dt_model.predict(X_test)
print("Decision Tree Classifier")
print(classification_report(y_test, dt_pred))
print(f'Accuracy: {accuracy_score(y_test, dt_pred)}')

Decision Tree Classifier
              precision    recall  f1-score   support

           0       0.98      0.99      0.99      2431
           1       0.99      0.98      0.99      2433

    accuracy                           0.99      4864
   macro avg       0.99      0.99      0.99      4864
weighted avg       0.99      0.99      0.99      4864

Accuracy: 0.9876644736842105

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV

# Define an expanded parameter grid
param_grid_rf = {
    'n_estimators': [200, 300],
    'max_features': ['sqrt', 'log2'],
    'max_depth': [8, 10],
    'criterion': ['gini', 'entropy']
}

# Initialize the model
rf_model = RandomForestClassifier(random_state=42)

# Perform Grid Search
grid_search_rf = GridSearchCV(estimator=rf_model, param_grid=param_grid_rf, cv=5, n_jobs=-1, verbose=2)
grid_search_rf.fit(X_train, y_train)

# Best parameters and score
print(f"Best parameters for Random Forest: {grid_search_rf.best_params_}")
print(f"Best score for Random Forest: {grid_search_rf.best_score_}")

# Predict and evaluate
rf_best_model = grid_search_rf.best_estimator_
rf_pred = rf_best_model.predict(X_test)
print("Random Forest Classifier after Hyperparameter Tuning")
print(classification_report(y_test, rf_pred))
print(f'Accuracy: {accuracy_score(y_test, rf_pred)}')

Fitting 5 folds for each of 16 candidates, totalling 80 fits

Best parameters for Random Forest: {'criterion': 'gini', 'max_depth': 10, 'max_features': 'log2', 'n_estimators': 300}
Best score for Random Forest: 0.9949105542485164
Random Forest Classifier after Hyperparameter Tuning
              precision    recall  f1-score   support

           0       1.00      0.99      1.00      2431
           1       0.99      1.00      1.00      2433

    accuracy                           1.00      4864
   macro avg       1.00      1.00      1.00      4864
weighted avg       1.00      1.00      1.00      4864

Accuracy: 0.9954769736842105

import lightgbm as lgb
lgb_model = lgb.LGBMClassifier(random_state=42)
lgb_model.fit(X_train, y_train)

# Predict and evaluate
lgb_pred = lgb_model.predict(X_test)
print("LightGBM Classifier")
print(classification_report(y_test, lgb_pred))
print(f'Accuracy: {accuracy_score(y_test, lgb_pred)}')

[LightGBM] [Info] Number of positive: 9725, number of negative: 9727
[LightGBM] [Info] Auto-choosing row-wise multi-threading, the overhead of testing was 0.003679 seconds.
You can set `force_row_wise=true` to remove the overhead.
And if memory is not enough, you can set `force_col_wise=true`.
[LightGBM] [Info] Total Bins 663
[LightGBM] [Info] Number of data points in the train set: 19452, number of used features: 49
[LightGBM] [Info] [binary:BoostFromScore]: pavg=0.499949 -> initscore=-0.000206
[LightGBM] [Info] Start training from score -0.000206

LightGBM Classifier
              precision    recall  f1-score   support

           0       1.00      0.99      1.00      2431
           1       0.99      1.00      1.00      2433

    accuracy                           1.00      4864
   macro avg       1.00      1.00      1.00      4864
weighted avg       1.00      1.00      1.00      4864

Accuracy: 0.9954769736842105

from sklearn.model_selection import GridSearchCV
import xgboost as xgb

# Define parameter grid
param_grid_xgb = {
    'n_estimators': [200,400],
    'max_depth': [8,10, 12],
    'learning_rate': [0.05, 0.1, 0.2],
}
# Initialize the model
xgb_model = xgb.XGBClassifier(random_state=42)

# Perform Grid Search
grid_search_xgb = GridSearchCV(estimator=xgb_model, param_grid=param_grid_xgb, cv=5, n_jobs=-1, verbose=2)
grid_search_xgb.fit(X_train, y_train)

# Best parameters and score
print(f"Best parameters for XGBoost: {grid_search_xgb.best_params_}")
print(f"Best score for XGBoost: {grid_search_xgb.best_score_}")

# Predict and evaluate
xgb_best_model = grid_search_xgb.best_estimator_
xgb_pred = xgb_best_model.predict(X_test)
print("XGBoost Classifier after Hyperparameter Tuning")
print(classification_report(y_test, xgb_pred))
print(f'Accuracy: {accuracy_score(y_test, xgb_pred)}')

Fitting 5 folds for each of 18 candidates, totalling 90 fits

Best parameters for XGBoost: {'learning_rate': 0.05, 'max_depth': 8, 'n_estimators': 200}
Best score for XGBoost: 0.9957330706481702
XGBoost Classifier after Hyperparameter Tuning
              precision    recall  f1-score   support

           0       1.00      0.99      1.00      2431
           1       0.99      1.00      1.00      2433

    accuracy                           1.00      4864
   macro avg       1.00      1.00      1.00      4864
weighted avg       1.00      1.00      1.00      4864

Accuracy: 0.99609375

!pip install keras-tuner > /dev/null 2>&1
import keras_tuner as kt
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.utils import to_categorical
from sklearn.metrics import classification_report, accuracy_score

# Convert target labels to one-hot encoding
y_train_onehot = to_categorical(y_train)
y_test_onehot = to_categorical(y_test)

# Build the neural network model
model = Sequential()
model.add(Dense(128, input_dim=X_train.shape[1], activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(y_train_onehot.shape[1], activation='softmax'))

# Compile the model
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

# Train the model
model.fit(X_train, y_train_onehot, epochs=150, batch_size=32, validation_split=0.2, verbose=2)

# Predict and evaluate
y_pred_proba = model.predict(X_test)
y_pred = y_pred_proba.argmax(axis=1)

print("Neural Network Classifier")
print(classification_report(y_test, y_pred))
print(f'Accuracy: {accuracy_score(y_test, y_pred)}')

Epoch 1/150

/opt/anaconda3/lib/python3.11/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

487/487 - 1s - 2ms/step - accuracy: 0.9088 - loss: 0.2071 - val_accuracy: 0.9900 - val_loss: 0.0426

Epoch 2/150

487/487 - 0s - 671us/step - accuracy: 0.9817 - loss: 0.0660 - val_accuracy: 0.9946 - val_loss: 0.0313

Epoch 3/150

487/487 - 0s - 725us/step - accuracy: 0.9876 - loss: 0.0507 - val_accuracy: 0.9951 - val_loss: 0.0288

Epoch 4/150

487/487 - 0s - 697us/step - accuracy: 0.9897 - loss: 0.0410 - val_accuracy: 0.9954 - val_loss: 0.0279

Epoch 5/150

import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc

def plot_multiple_roc_curves(y_test, y_pred_probas, model_names):
    plt.figure(figsize=(12, 8))

    for y_pred_proba, model_name in zip(y_pred_probas, model_names):
        fpr, tpr, _ = roc_curve(y_test, y_pred_proba)
        roc_auc = auc(fpr, tpr)
        plt.plot(fpr, tpr, lw=2, label=f'{model_name} (AUC = {roc_auc:.2f})')

    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver Operating Characteristic')
    plt.legend(loc="lower right")
    plt.show()

y_pred_probas = [nb_pred, lr_pred, dt_pred, rf_pred, lgb_pred, xgb_pred, y_pred]
model_names = ['Naive Bayes', 'Logistic Regression', 'Decision Tree', 'Random Forest', 'LightGBM', 'XGBoost', 'Neural Network']

# Plot ROC curves
plot_multiple_roc_curves(y_test, y_pred_probas, model_names)

!pip install shap > /dev/null 2>&1
!pip install shap xgboost  > /dev/null 2>&1

import shap
explainer = shap.TreeExplainer(xgb_best_model)
X_test_df = X_test if isinstance(X_test, pd.DataFrame) else pd.DataFrame(X_test)
shap_values = explainer.shap_values(X_test_df)
shap_values = shap_values if isinstance(shap_values, list) else [shap_values]

shap.summary_plot(shap_values[0], X_test_df)

# Bar plot for feature importance
shap.summary_plot(shap_values[0], X_test_df, plot_type="bar")

# Force plot for individual prediction
shap.initjs()
shap.force_plot(explainer.expected_value, shap_values[0][0].reshape(1, -1), X_test_df.iloc[0].values.reshape(1, -1))
actual_feature_name = X_test_df.columns[0]  # Replace with the actual feature name you want to plot
# Dependence plot
shap.dependence_plot(actual_feature_name, shap_values[0], X_test_df)

indices_of_interest = [1, 27, 28]
X_selected.columns[indices_of_interest]

Index(['Number_of_casualties',
       'Lanes_or_Medians_Two-way (divided with broken lines road marking)',
       'Lanes_or_Medians_Undivided Two way'],
      dtype='object')

	Time	Day_of_week	Age_band_of_driver	Sex_of_driver	Educational_level	Vehicle_driver_relation	Driving_experience	Type_of_vehicle	Owner_of_vehicle	Service_year_of_vehicle	...	Vehicle_movement	Casualty_class	Sex_of_casualty	Age_band_of_casualty	Casualty_severity	Work_of_casuality	Fitness_of_casuality	Pedestrian_movement	Cause_of_accident	Accident_severity
0	17:02:00	Monday	18-30	Male	Above high school	Employee	1-2yr	Automobile	Owner	Above 10yr	...	Going straight	na	na	na	na	NaN	NaN	Not a Pedestrian	Moving Backward	Slight Injury
1	17:02:00	Monday	31-50	Male	Junior high school	Employee	Above 10yr	Public (> 45 seats)	Owner	5-10yrs	...	Going straight	na	na	na	na	NaN	NaN	Not a Pedestrian	Overtaking	Slight Injury
2	17:02:00	Monday	18-30	Male	Junior high school	Employee	1-2yr	Lorry (41?100Q)	Owner	NaN	...	Going straight	Driver or rider	Male	31-50	3	Driver	NaN	Not a Pedestrian	Changing lane to the left	Serious Injury
3	1:06:00	Sunday	18-30	Male	Junior high school	Employee	5-10yr	Public (> 45 seats)	Governmental	NaN	...	Going straight	Pedestrian	Female	18-30	3	Driver	Normal	Not a Pedestrian	Changing lane to the right	Slight Injury
4	1:06:00	Sunday	18-30	Male	Junior high school	Employee	2-5yr	NaN	Owner	5-10yrs	...	Going straight	na	na	na	na	NaN	NaN	Not a Pedestrian	Overtaking	Slight Injury

	Time	Day_of_week	Age_band_of_driver	Sex_of_driver	Educational_level	Vehicle_driver_relation	Driving_experience	Type_of_vehicle	Owner_of_vehicle	Service_year_of_vehicle	...	Number_of_vehicles_involved	Number_of_casualties	Vehicle_movement	Casualty_severity	Work_of_casuality	Fitness_of_casuality	Pedestrian_movement	Cause_of_accident	Accident_severity	hour
0	1900-01-01 17:02:00	Monday	18-30	Male	Above high school	Employee	1-2yr	Automobile	Owner	Above 10yr	...	1.098612	1.098612	Going straight	na	Driver	Unknown	Not a Pedestrian	Moving Backward	Slight Injury	17
1	1900-01-01 17:02:00	Monday	31-50	Male	Junior high school	Employee	Above 10yr	Public (> 45 seats)	Owner	5-10yrs	...	1.098612	1.098612	Going straight	na	Driver	Unknown	Not a Pedestrian	Overtaking	Slight Injury	17
2	1900-01-01 17:02:00	Monday	18-30	Male	Junior high school	Employee	1-2yr	Lorry (41?100Q)	Owner	Unknown	...	1.098612	1.098612	Going straight	3	Driver	Unknown	Not a Pedestrian	Changing lane to the left	Serious Injury	17
3	1900-01-01 01:06:00	Sunday	18-30	Male	Junior high school	Employee	5-10yr	Public (> 45 seats)	Governmental	Unknown	...	1.098612	1.098612	Going straight	3	Driver	Normal	Not a Pedestrian	Changing lane to the right	Slight Injury	1
4	1900-01-01 01:06:00	Sunday	18-30	Male	Junior high school	Employee	2-5yr	Automobile	Owner	5-10yrs	...	1.098612	1.098612	Going straight	na	Driver	Unknown	Not a Pedestrian	Overtaking	Slight Injury	1

Project Description: Urban Traffic Incident Detection Using Real-Time Sensor Data¶

Dataset Column Descriptions¶

Import Library¶

Data¶

Explatory Data Analysis¶

Feature Engineering¶

Time-based features¶

Encoding categorical variables¶

Model selection¶

Naive Bayes Classifier¶

Logistic Regression¶

Decision Tree¶

Random Forest¶

Light GBM¶

XGBoost¶

Neural Network¶

Plots for Receiver Operating Characteristic (ROC)¶

SHAP values¶

1. SHAP Summary Plot (Beeswarm Plot)¶

2. SHAP Bar Plot (Feature Importance)¶

3. SHAP Dependence Plot¶

To summarize:¶

	Data Type	Missing Values	Unique Values
Defect_of_vehicle	object	4427	3
Service_year_of_vehicle	object	3928	6
Work_of_casuality	object	3198	7
Fitness_of_casuality	object	2635	5
Type_of_vehicle	object	950	17
Types_of_Junction	object	887	8
Driving_experience	object	829	7
Educational_level	object	741	7
Vehicle_driver_relation	object	579	4
Owner_of_vehicle	object	482	4
Lanes_or_Medians	object	385	7
Vehicle_movement	object	308	13
Area_accident_occured	object	239	14
Road_surface_type	object	172	5
Type_of_collision	object	155	10
Road_allignment	object	142	9
Casualty_class	object	0	4
Pedestrian_movement	object	0	9
Cause_of_accident	object	0	20
Casualty_severity	object	0	4
Age_band_of_casualty	object	0	6
Sex_of_casualty	object	0	3
Time	object	0	1074
Road_surface_conditions	object	0	4
Number_of_casualties	int64	0	8
Number_of_vehicles_involved	int64	0	6
Weather_conditions	object	0	9
Light_conditions	object	0	4
Day_of_week	object	0	7
Sex_of_driver	object	0	3
Age_band_of_driver	object	0	5
Accident_severity	object	0	3

	Number_of_vehicles_involved	Number_of_casualties	hour	Day_of_week_Saturday	Day_of_week_Sunday	Day_of_week_Thursday	Age_band_of_driver_31-50	Age_band_of_driver_Over 51	Educational_level_Elementary school	Educational_level_High school	...	Vehicle_movement_Going straight	Casualty_severity_3	Casualty_severity_na	Work_of_casuality_Self-employed	Fitness_of_casuality_Normal	Fitness_of_casuality_Unknown	Cause_of_accident_Changing lane to the right	Cause_of_accident_Driving carelessly	Cause_of_accident_Moving Backward	Cause_of_accident_No distancing
0	1.098612	1.098612	17	False	False	False	False	False	False	False	...	True	False	True	False	False	True	False	False	True	False
1	1.098612	1.098612	17	False	False	False	True	False	False	False	...	True	False	True	False	False	True	False	False	False	False
2	1.098612	1.098612	17	False	False	False	False	False	False	False	...	True	True	False	False	False	True	False	False	False	False
3	1.098612	1.098612	1	False	True	False	False	False	False	False	...	True	True	False	False	True	False	True	False	False	False
4	1.098612	1.098612	1	False	True	False	False	False	False	False	...	True	False	True	False	False	True	False	False	False	False